﻿#coding=UTF-8
import datetime
import re
import HTMLParser
import logging
log = logging.getLogger("appparser")

HookUpdateGlobalDate = None

class URLParser(HTMLParser.HTMLParser):
  def __init__(self, url):
    self.reset()
    self.root = url[0:url.rfind("/")]
    HTMLParser.HTMLParser.__init__(self)
    
  def unknown_decl(self, data):
    return
        
  def reset(self):
    HTMLParser.HTMLParser.reset(self)
    self.urls = []
        
  def handle_starttag(self, tag, attrs):
    if "a" != tag.lower(): return
    dAttrs = dict(attrs)
    if not dAttrs.has_key("href"): return
    
    url = self.join(dAttrs["href"])
    if url:
      self.urls.append(url)
  
  def join(self, url):
    if not re.match(".*\.(htm)|(html)]$", url, re.IGNORECASE):
      return ""
    
    if re.match("^http://.*", url, re.IGNORECASE):
      return url
    if url[0] != "/": url = "/" + url
    return self.root + url

class TableParser(HTMLParser.HTMLParser):
  tblTags = ["table", "th", "tr", "td", "caption", "thead", "tbody", "tfoot", "span", "head", "style"]
  patterns = ["table/caption",
              "table/tr/th",
              "table/thead/tr/th",
              "table/tr/td",
              "table/tbody/tr/td",
              "table/tfoot/tr/td"]
  indentPattern = "table/tr/td/span"
  
  def __init__(self, url=""):
    self.source = url
    self.records = []
    self.tableStack = []
    self.style = {}
    self.reset2()
    HTMLParser.HTMLParser.__init__(self)
    
  def unknown_decl(self, data):
    return

  def handle_entityref(self, name):
    if "/".join(self.tagStack) == self.indentPattern \
    and "nbsp" == name \
    and not self.data:
      self.dataIndent += 1
      return

  def handle_data(self, data):
    if "/".join(self.tagStack) == "head/style":
      self.saveStyleInfo(data)
      return
    if "/".join(self.tagStack) not in self.patterns \
    or not len(data.strip()):
      return
    self.data += data.strip()
        
  def reset(self):
    HTMLParser.HTMLParser.reset(self)
    self.records = []
    self.tableStack = []
    self.dates = Dates()
    self.reset2()
    
  def reset2(self):
    self.tagStack = []
    self.data = ""
    self.dataIndent = 0
    self.tagLevel = None
    self.tagLevels = [TagLevel1(), TagLevel2(), TagLevel()]
    self.table = Table()
    self.units = Units()
    self.tags = Tags()
        
  def handle_starttag(self, tag, attrs):
    if tag not in self.tblTags:
      return
    
    if "table" == tag.lower() \
    and len(self.tagStack):
      self.tableStack.append((self.tagStack, self.tagLevel, self.table,
                              self.units, self.tags))
      self.reset2()
      
    self.tagStack.append(tag.lower())
    if tag.lower() == "tr":
      self.table.currCell.col = 0
    
    if tag.lower() != "td" \
    and tag.lower() != "th":
      return

    self.table.setCellInfo(attrs)
    self.setDataIndent(attrs)
    
  def handle_endtag(self, tag):
    if tag not in self.tblTags \
    or not len(self.tagStack) \
    or tag != self.tagStack[-1]:
      return
    
    self.tagStack.pop()
    if "table" == tag.lower() \
    and len(self.tableStack):
      prevTb = self.tableStack.pop()
      self.tagStack = prevTb[0]
      self.tagLevel = prevTb[1]
      self.table = prevTb[2]
      self.units = prevTb[3]
      self.tags = prevTb[4]

    if ("td" == tag.lower() \
    or "th" == tag.lower()) \
    and self.table.isSingleCell():
      self.filterCell(re.sub("\s", "", self.data).strip())

    if "tr" == tag.lower():
      self.table.incCurrRow()
    if ("td" == tag.lower() \
    or "th" == tag.lower()):
      self.table.incCurrCol()
      self.dataIndent = 0
      self.data = ""
    
  def filterCell(self, data):
    if not data.strip(): return
    data = self.dates.filterDate(self.table.currCell, data)
    if not data.strip(): return
    data = self.units.filterUnit(self.table.currCell, data)
    if not data.strip(): return

    data, unit = self.filterRecord(data)
    if not data.strip(): return
    if unit:
      self.records.append((data, unit, self.dates.getDate(self.table.currCell),
                           self.tags.getTags(self.table.currCell)))          
    else:
      if self.hasTagLevel():
        if self.tagLevel is None:
          for tagLevel in self.tagLevels:
            data, matched = tagLevel.check(self, data, self.dataIndent)
            if matched:
              self.tagLevel = tagLevel
              break
        else:
          data, matched = self.tagLevel.check(self, data, self.dataIndent)
      self.tags.addTags(self.table.currCell, data)
      if 0 == self.table.currCell.col \
      and 1 == self.table.currCell.colSpan \
      and self.dates.updateGlobalDate(self.table.currCell.row) \
      and HookUpdateGlobalDate is not None: 
        HookUpdateGlobalDate(self)
  
  def filterRecord(self, data):
    rData = "(?P<data>[+-]?\d+(\.\d*)?)"
    rRec = ["^%s?%s$" %(self.units.base, rData),
            "^%s%s?$" %(rData, self.units.base),]
    for r in rRec:
      m = re.match(r, data)
      if m:
        if m.group("unit"):
          return m.group("data"), m.group("unit")
        else:
          return m.group("data"), self.units.getUnit(self.table.currCell, self.dates)
    return data, ""
  
  def hasBlankDate(self):
    return  len(self.records) > 0 \
            and  ((self.dates.hasRowDate() \
            and len(self.dates.colDates) > len(self.dates.colDateHits)\
            and not (len(self.dates.colDates) == 1 \
                 and self.dates.colDates.values()[0] == self.dates.date)
            ) \
            or (self.dates.hasColDate() \
            and len(self.dates.rowDates) > len(self.dates.rowDateHits)\
            and not (len(self.dates.rowDates) == 1 \
                 and self.dates.rowDates.values()[0] == self.dates.date)
            ))
            
  def saveStyleInfo(self, data):
    rSplit = "(\.[a-z]+[0-9]+[^{^}]*{[^{^}]*})"
    data = re.compile(rSplit, re.DOTALL | re.MULTILINE | re.IGNORECASE | re.UNICODE).split(data)
    
    rStyle = ".*((?P<indentname>xl[1-9][0-9]{1,6}).*{.*mso-char-indent-count:(?P<indentvalue>[1-9]+);.*}).*"
    for section in data:
      m = re.match(rStyle, section, re.DOTALL | re.MULTILINE | re.IGNORECASE | re.UNICODE)
      if not m: continue
      self.style[m.group("indentname")] = int(m.group("indentvalue"))

  def setDataIndent(self, attrs):
    dAttrs = dict(attrs)
    if dAttrs.has_key("class") \
    and self.style.has_key(dAttrs["class"]):
      self.dataIndent = self.style[dAttrs["class"]]-1
    else:
      self.dataIndent = 0
      
  def hasTagLevel(self):
    return 0 != self.dates.dateCol \
            and self.table.currCell.row != self.dates.dateRow
            
  def checkDateUnit(self):
    if not self.dates.hasDate:
      log.warning("URL[%s] has no date." %(self.source))
    if not self.units.hasUnit:
      log.warning("URL[%s] has no unit." %(self.source))

class Cell(object):
  def __init__(self):
    self.col = 0
    self.row = 0
    self.colSpan = 1
    self.rowSpan = 1
    
class Table(object):
  def __init__(self):
    self.currCell = Cell()
    self.cells = {}

  def setCellInfo(self, attrs):
    dAttrs = dict(attrs)
    if dAttrs.has_key("colspan"):
      self.currCell.colSpan = int(dAttrs["colspan"])
    else:
      self.currCell.colSpan = 1
    if dAttrs.has_key("rowspan"):
      self.currCell.rowSpan = int(dAttrs["rowspan"])
    else:
      self.currCell.rowSpan = 1

    while self.fillCell() is "Merged":
      self.incCurrCol()

  def fillCell(self):
    if self.cells.has_key(self.currCell.row) \
    and self.cells[self.currCell.row].has_key(self.currCell.col):
      assert self.cells[self.currCell.row][self.currCell.col] is "Merged"
      return "Merged"
    i = 0
    while i<self.currCell.rowSpan:
      j = 0
      while j<self.currCell.colSpan:
        if not self.cells.has_key(self.currCell.row+i):
          self.cells[self.currCell.row+i] = {}
        self.cells[self.currCell.row+i][self.currCell.col+j] = "Merged"
        j+=1
      i+=1
    self.cells[self.currCell.row][self.currCell.col] = "Single"
    return "Single"

  def isSingleCell(self):
    return self.cells[self.currCell.row][self.currCell.col] is "Single"
  
  def incCurrCol(self, inc=1):
    self.currCell.col += inc
  
  def incCurrRow(self, inc=1):
    self.currCell.row += inc
    
class Dates(object):
  def __init__(self):
    self.date = (datetime.datetime.now(), "%Y%m%d")
    self.colDates = {}
    self.rowDates = {}
    self.colDateHits = {}
    self.rowDateHits = {}
    self.dateCol = -1
    self.dateRow = -1
    self.hasDate = False
    
  def hasColDate(self):
    return self.dateCol >= 0
  
  def hasRowDate(self):
    return self.dateRow >= 0
  
  def getDate(self, cell):
    if self.hasColDate() \
    and self.rowDates.has_key(cell.row):
      self.rowDateHits[cell.row] = True
      return self.rowDates[cell.row]
    if self.hasRowDate() \
    and self.colDates.has_key(cell.col):
      self.colDateHits[cell.col] = True
      return self.colDates[cell.col]
    return self.date

  def filterDate(self, cell, data):
    matched, date, format = IsDate(data, self.date[0])
    if matched:
      data = re.sub(matched.group("date"), "", data)
      if not self.hasColDate() \
      and not self.hasRowDate():
        self.date = (date, format)
        if cell.colSpan == 1 \
        and cell.rowSpan == 1:
          self.dateCol = cell.col
          self.dateRow = cell.row
          self.rowDates[cell.row] = (date, format)
          self.colDates[cell.col] = (date, format)
          self.hasDate = True            
      elif cell.col == self.dateCol:
        self.rowDates[cell.row] = (date, format)
        self.dateRow = -1
        self.hasDate = True
      elif cell.row == self.dateRow:
        self.colDates[cell.col] = (date, format)
        self.dateCol = -1
        self.hasDate = True 
    return data
  
  def updateGlobalDate(self, row):
    if self.hasColDate() \
    and 0 == self.dateCol:
      while True:
        row -= 1
        if self.rowDates.has_key(row):
          self.date = self.rowDates[row]
          return True
    return False
  
def IsDate(data, globalDate):
  rY = "19[7-9][0-9]|20[0-2][0-9]"
  rM = "1[0-2]|0?[1-9]"
  rD = "[1-2][0-9]|3[0-1]|0?[1-9]"
  uY = u"[年]"
  uM = u"[月]"
  uD = u"[日|号]"
  prefix = ".*[^\-\+0-9]+"
  rYear = [(prefix + "(?P<date>" + "(" + rY + ")"+uY+"(" + rM + ")"+uM+"(" + rD + ")"+uD + ")", _replNormal),
           (prefix + "(?P<date>" + "(" + rY + ")[.-](" + rM + ")[.-](" + rD + ")" + ")", _replNormal),
           (prefix + "(?P<date>" + "(" + rY + ")"+uY+"(" + rM + ")"+uM + ")", _replNormal),
           (prefix + "(?P<date>" + "(" + rY + ")[.-](" + rM + ")" + ")", _replNormal),
           (prefix + "(?P<date>" + "(" + rY + ")" +uY + ")", _replNormal),
           (prefix + "(?P<date>" + "(" + rY + ")$" + ")", _replNormal),
           ("(?P<date>" + "(" + rY + ")"+uY+"(" + rM + ")"+uM+"(" + rD + ")"+uD + ")", _replNormal),
           ("(?P<date>" + "(" + rY + ")[.-](" + rM + ")[.-](" + rD + ")" + ")", _replNormal),
           ("(?P<date>" + "(" + rY + ")"+uY+"(" + rM + ")"+uM + ")", _replNormal),
           ("(?P<date>" + "(" + rY + ")[.-](" + rM + "$)" + ")", _replNormal),
           ("(?P<date>" + "(" + rY + ")" +uY + ")", _replNormal),
           ("(?P<date>" + "(" + rY + ")$" + ")", _replNormal),
           ]
  rMonth = [(prefix + "(?P<date>" + "(" + rM + ")" + uM + "("+ rD + ")" +uD + ")", _replNormal),
#             (prefix + "(?P<date>" + "(" + rM + ")[.-](" + rD + ")" + ")", _replNormal),
            (prefix + "(?P<date>" + "(" + rM + ")" +uM + ")", _replNormal),
            ("(?P<date>" + "(" + rM + ")" + uM + "("+ rD + ")" +uD + ")", _replNormal),
#            ("(?P<date>" + "(" + rM + ")[.-](" + rD + ")" + ")", _replNormal),
            ("(?P<date>" + "(" + rM + ")" +uM + ")", _replNormal),
            (u"(?P<date>[一|二|三|四])(季度)", _replQuarter),
            ]
  rDay = [(prefix + "(?P<date>" + "(" + rD +")" +uD + ")", _replNormal),
          ("(?P<date>" + "(" + rD +")" +uD + ")", _replNormal),
          ]
  yFmt = {1:"%Y",
          2:"%Y%m",
          3:"%Y%m%d"}
  mFmt = {1:"%Y%m",
          2:"%Y%m%d"}
  dFmt = {1:"%Y%m%d"}
  matched, date, format = _IsDate(rYear, data, "", yFmt)
  if matched is None:
    matched, date, format = _IsDate(rMonth, data, globalDate.strftime("%Y"), mFmt)
  if matched is None:
    matched, date, format = _IsDate(rDay, data, globalDate.strftime("%Y%m"), dFmt)
  return matched, date, format

def _replNormal(date, prefix, ymd, fmt):
  ymd.remove(date)
  return datetime.datetime.strptime(prefix+"".join(ymd), fmt[len(ymd)]), fmt[len(ymd)]
  
def _replQuarter(date, prefix, ymd, fmt):
  q = {u"一":"0331",
       u"二":"0630",
       u"三":"0930",
       u"四":"1231",
       }
  return datetime.datetime.strptime(prefix+q[date], "%Y%m%d"), "%Y%m%d"
  
def _IsDate(rDate, data, prefix, fmt):
  for r, repl in rDate:
    m = re.match(r, data)
    if m:
      ymd = list(m.groups())
      date, format = repl(m.group("date"), prefix, ymd, fmt)
      if "%Y" == format:
        date = datetime.datetime.strptime("%s1231" %(date.year), "%Y%m%d")
      return m, date, format
  return None, None, None

class Units(object):
  def __init__(self):
    self.unit = u"单位"
    self.base = u"(?P<unit>\D*[%|\uff05|‰|元|个|所|座|吨|米|时|人|对|台|户|册|部|张|里|辆|起|度|顷|瓦|斤|股|万手|门|路端])"
    self.colUnits = {}
    self.rowUnits = {}
    self.hasUnit = False
    
  def getUnit(self, cell, dates):
    if dates.hasColDate():
      if self.colUnits.has_key(cell.col):
        return self.colUnits[cell.col]
    if dates.hasRowDate():
      if self.rowUnits.has_key(cell.row):
        return self.rowUnits[cell.row]
    
    if self.colUnits.has_key(cell.col):
      return self.colUnits[cell.col]
    if self.rowUnits.has_key(cell.row):
      return self.rowUnits[cell.row]
    
    if self.unit: return self.unit
    
    if self.colUnits.has_key(cell.col-1):
      self.colUnits[cell.col] = self.colUnits[cell.col-1]
      return self.colUnits[cell.col]
    if self.rowUnits.has_key(cell.row-1):
      self.rowUnits[cell.row] = self.rowUnits[cell.row-1]
      return self.rowUnits[cell.row]

  def filterUnit(self, cell, data):
    rUnitGlobal = [
             (u"\D*(?P<global>单位[：|:](?P<unit>.*))", self.replGlobal),
             (u"(?P<unit>以上年同月为100)", self.replNormal),
             ]
    rUnit = [
             (u"\D*\(%s\)$" %(self.base), self.replNormal),
             (u"\D*(?P<unit>\(\D*?[=|＝]100\D*?\)$)", self.replNormal),
#             (u"(?P<unit>\D*?[=|＝]1\d*\D*?$)", self.replNormal),
             (u".*(?P<unit>\([0-9]*年[=|＝]100\)$)", self.replNormal),
             (u"\D*(?P<unit>\(.*?[吨]\D*?\)$)", self.replNormal),
             (u"\D*(?P<unit>[\(|（]人民币[/|／]\D*[\)|）]$)", self.replNormal),
             ]
    
    for r, repl in rUnitGlobal+rUnit:
      m = re.match(r, data)
      if m:
        u = re.sub(u"[\(|\)|（|）]", "", m.group("unit"))
        if dict(rUnitGlobal).has_key(r):
          if not self.unit: self.unit = u
        else:
          self.rowUnits[cell.row] = u
          self.colUnits[cell.col]= u
        self.hasUnit = True
        return repl(data, m)
    return data
  
  def replNormal(self, data, m):
    data = re.sub(u"[\(|\)|（|）]", "", data)
    return re.sub(re.sub(u"[\(|\)|（|）]", "", m.group("unit")), "", data)
  
  def replGlobal(self, data, m):
    data = re.sub(m.group("global"), "", data)
    return re.sub(u"[\(|\)|（|）]", "", data)

class Tags(object):
  def __init__(self):
    self.globalTags = []
    self.colTags = {}
    self.rowTags = {}

  def getTags(self, cell):
    tags = cTags = rTags = []    
    if self.colTags.has_key(cell.col) \
    and len(self.colTags[cell.col]):
      cTags = dict(self.colTags[cell.col]).keys()
    if self.rowTags.has_key(cell.row) \
    and len(self.rowTags[cell.row]):
      rTags = dict(self.rowTags[cell.row]).keys()
    tags = cTags + list(set(rTags)-set(cTags))
    tags += list(set(self.globalTags)-set(tags))
    return tags
  
  def addTag(self, tags, index, data, span, append=False):
    data = self.refineTag(data)
    if not data.strip(): return
    if not tags.has_key(index):
      tags[index] = []
    if append \
    and 1 == span\
    and len(tags[index]):
      last = tags[index].pop()
      if 1 == last[1][0] \
      and 0 != last[1][1]:
        data = last[0] + data
        data = self.refineTag(data)
      else:
        tags[index].append(last)
    if not dict(tags[index]).has_key(data):
      tags[index].append((data, (span, index)))
    
  def addTags(self, cell, data):
    self.addTag(self.colTags, cell.col, data, cell.colSpan, True)
    if cell.colSpan > 1:
      i = 1
      while i < cell.colSpan:
        self.addTag(self.colTags, cell.col+i, data, cell.colSpan)
        i+=1
    self.addTag(self.rowTags, cell.row, data, cell.rowSpan)
    if cell.rowSpan > 1:
      i = 1
      while i < cell.rowSpan:
        self.addTag(self.rowTags, cell.row+i, data, cell.rowSpan)
        i+=1
        
  def refineTag(self, data):
    rExceptions = [u"(\d+-)*\d+米.*",
              u"[A|a|B|b]股",
              u"\d*[天|个月|年]",
              ]
    for r in rExceptions:
      if re.match(r, data):
        return data
    
    rTag = [u"[。≥≤]",
            u"本表.*",
            u"其中[：|:]",
            u"（净）",
            u"[\s\d\-\(\).#%=a-zA-Z:：（）、％,／①②]",
            ]
    for r in rTag:
      data = re.sub(r, "", data)
    
    return data

class TagLevel(object):
  def __init__(self):
    self.tagStack = [] #[(tag, level, level-1.index)]
    
  def check(self, dataparser, data, dataIndent):
    if dataparser.table.currCell.col > 0: return data, False
    data, level = self.getLevel(data, dataIndent)
    self.addTag(dataparser, data, dataIndent if 99 == level else level)
    return data, level != 99
    
  def getLevel(self, data, dataIndent):
    return data, 99
  
  def addTag(self, dataparser, data, level):
    if 0 == level:
      self.tagStack = []
      self.tagStack.append((data, level))
    elif len(self.tagStack):
      while True:
        last = self.tagStack[-1]
        if last[1] < level:
          self.tagStack.append((data, level))
          break
        else:
          self.tagStack.pop()
    if level and len(self.tagStack)>1:
      last = len(self.tagStack)-2
      while True:
        dataparser.tags.addTag(dataparser.tags.rowTags, dataparser.table.currCell.row,
                               self.tagStack[last][0], dataparser.table.currCell.rowSpan)
        last -= 1
        if -1 == last: break
  
class TagLevel1(TagLevel):
  def getLevel(self, data, dataIndent):
    rLevel = [u"([一|二|三|四|五|六|七|八|九|十]+\.)", #level 0
              u"([A-Z]+\.)", #level 1
              u"([a-z]+\.)", #level 2
              u"([0-9]+(\.[0-9])+)", #level 4+
              u"([0-9]+\.)", #level 3
              ]
    level = 99
    for r in rLevel:
      m = re.match(r, data)
      if m:
        if 3 == rLevel.index(r):
          level = m.groups()[0].count(".") + 3
        elif 4 == rLevel.index(r):
          level = 3
        else:
          level = rLevel.index(r)
        return re.sub(r, "", data), level
    return data, level
  
class TagLevel2(TagLevel):
  def check(self, dataparser, data, dataIndent):
    data, level = self.getLevel(data, dataIndent)
    self.addTag(dataparser, data, dataIndent+level if 99 == level else level)
    return data, level != 99

  def getLevel(self, data, dataIndent):
    rLevel = [u"([一|二|三|四|五|六|七|八|九|十]+、)", #level 0
              u"([0-9]+\.)", #level 1
              u"(\([0-9]+\))", #level 2
              ]
    level = 99
    for r in rLevel:
      m = re.match(r, data)
      if m:
        level = rLevel.index(r)
        return re.sub(r, "", data), level
    return data, level
